package com.webull.information.center.carwler.task.handler;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Queue;
import java.util.TimeZone;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.JsonNode;
import com.github.stuxuhai.jpinyin.ChineseHelper;
import com.webull.framework.util.UtilDate;
import com.webull.information.center.carwler.common.json.unit.GetNewsStory_1;
import com.webull.information.center.carwler.common.json.unit.GetTopNewsEntitledColumns_1;
import com.webull.information.center.carwler.common.json.unit.News_1_RetrieveHeadlineML_1;
import com.webull.information.center.carwler.common.json.unit.News_1_RetrieveStoryML_1;
import com.webull.information.center.carwler.common.json.unit.RetrieveTopNewsByColumnID_1;
import com.webull.information.center.carwler.common.model.ArchiveRel;
import com.webull.information.center.carwler.common.model.NewsInformation;
import com.webull.information.center.carwler.common.model.StockTitle;
import com.webull.information.center.carwler.common.model.TickerChain;
import com.webull.information.center.carwler.common.util.HttpClientUtil;
import com.webull.information.center.carwler.common.util.NewsCcUtil;
import com.webull.information.center.carwler.common.util.ReutersGuideUtil;
import com.webull.information.center.carwler.common.util.aws.s3.AwsS3ClientUtil;
import com.webull.information.center.carwler.common.util.aws.s3.AwsUsS3ClientUtil;
import com.webull.information.center.carwler.common.util.news.ReutersTopNewsListUitl;
import com.webull.information.center.carwler.dao.ArchiveDao;
import com.webull.information.center.carwler.task.support.StockConfigSupport;
import com.webull.information.center.carwler.task.support.TikcerNewsProcessor;
import com.webull.information.center.common.constants.Constants;
import com.webull.information.center.common.util.SimilarDegreeByCos;
import com.webull.information.client.model.News;
import com.webull.information.client.model.parm.NewsRelType;
import com.webull.securities.client.model.Currency;
import com.webull.securities.client.model.Fund;
import com.webull.securities.client.model.Index;
import com.webull.securities.client.model.Ticker;
import com.webull.securities.client.service.ExchangeService;

/**
 * 路透guide新闻处理
 *
 * @author shimingjun
 * @version 1.0
 * @date 2016年8月26日 下午4:06:09
 * @since JDK 1.8
 */
@Component
public class StockReuteGuideHandler {

    protected final Logger logger = LogManager.getLogger(getClass());
    private @Autowired
    TikcerNewsProcessor tikcerNewsProcessor;
    private @Autowired
    ReutersGuideUtil reutersGuideUtil;
    private @Autowired
    ArchiveDao archiveDao;
    private @Autowired
    ExchangeService exchangeService;
    private @Autowired
    HttpClientUtil httpClientUtil;
    private @Autowired
    AwsS3ClientUtil awsS3ClientUtil;
    private @Autowired
    AwsUsS3ClientUtil awsUsS3ClientUtil;
	private @Autowired StockConfigSupport stockConfigSupport;
	private final int maxCount = 25;

    public void handlerByStockTitle4Reuters(StockTitle stockTitle) {

        try {
            ArchiveRel archive = archiveDao.getArchiveRel(Ticker.TickerTypeConstants.STOCK, stockTitle.getTickerId());

            String nRic = Optional.ofNullable(archive)
                    .map(archive0 -> Optional.ofNullable(archive0.getRtNricId()).orElse(archive0.getRtRicId()))
                    .orElse(null);
            // nRic = "7270.T";
            if (StringUtils.isBlank(nRic))
                return;

            /**
             * 获取标的物新闻首页的列表2
             */
			String jsonParam2 = News_1_RetrieveHeadlineML_1.getRicRequestJson(maxCount, nRic);
            JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam2);
            ProcessNewsParm parm = new ProcessNewsParm();
            parm.exchangeId = stockTitle.getExchangeId();
            parm.regionId = stockTitle.getRegionId();
            parm.targetId = stockTitle.getTickerId();
			processReutersNews(jsonNode2, parm);


        } catch (Exception e) {

        }

    }

    public void handlerByFund4Reuters(Fund stockTitle) {

        try {
            ArchiveRel archive = archiveDao.getArchiveRel(Ticker.TickerTypeConstants.FUND, stockTitle.getTickerId());

            String nRic = Optional.ofNullable(archive)
                    .map(archive0 -> Optional.ofNullable(archive0.getRtNricId()).orElse(archive0.getRtRicId()))
                    .orElse(null);
            if (StringUtils.isBlank(nRic))
                return;

            /**
             * 获取标的物新闻首页的列表2
             */
			String jsonParam2 = News_1_RetrieveHeadlineML_1.getRicRequestJson(maxCount, nRic);
            JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam2);
            ProcessNewsParm parm = new ProcessNewsParm();
            parm.exchangeId = stockTitle.getExchangeId();
            parm.regionId = stockTitle.getRegionId();
            parm.targetId = stockTitle.getTickerId();
			processReutersNews(jsonNode2, parm);

        } catch (Exception e) {

        }

    }

    public void handlerByIndexTitle4Reuters(Index index) {

        try {
            ArchiveRel archive = archiveDao.getArchiveRel(Ticker.TickerTypeConstants.INDEX, index.getTickerId());

            String nRic = Optional.ofNullable(archive)
                    .map(archive0 -> Optional.ofNullable(archive0.getRtNricId()).orElse(archive0.getRtRicId()))
                    .orElse(null);
            // nRic = ".NSEI";
            if (StringUtils.isBlank(nRic))
                return;

            /**
             * 获取标的物新闻首页的列表2
             */
			String jsonParam2 = News_1_RetrieveHeadlineML_1.getRicRequestJson(maxCount, nRic);
            JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam2);
            ProcessNewsParm parm = new ProcessNewsParm();
            parm.exchangeId = index.getExchangeId();
            parm.regionId = Optional.ofNullable(parm.exchangeId).map(id0 -> exchangeService.getById(id0))
                    .map(ex0 -> ex0.getRegionId()).orElse(null);
            parm.targetId = index.getTickerId();
			processReutersNews(jsonNode2, parm);


        } catch (Exception e) {
            logger.error("handlerByIndexTitle4Reuters index :{},error:{}", index, e);
        }

    }

    /**
     * TODO
     *
     * @param tickerChain
     * @since 0.1.0
     */
    public void handlerByFutrueTitle4Reuters(TickerChain tickerChain) {
        try {
            String nRic = Optional.ofNullable(tickerChain).map(archive0 -> tickerChain.getRtNricId()).orElse("");
            if (StringUtils.isBlank(nRic))
                return;
            /**
             * 获取标的物新闻首页的列表2
             */

            Optional.ofNullable(archiveDao.getArchiveRelBySrcId(ArchiveRel.Src.rt_chain, tickerChain.getChainId()))//
                    .filter(ar -> ArchiveRel.Type.group.equals(ar.getType()))//
                    .map(ar0 -> ar0.getWlId())//
                    .ifPresent(groupsId -> {
                        ProcessNewsParm parm = new ProcessNewsParm();
                        parm.exchangeId = tickerChain.getExchangeId();
                        parm.regionId = Optional.ofNullable(parm.exchangeId).map(id0 -> exchangeService.getById(id0))
                                .map(ex0 -> ex0.getRegionId()).orElse(null);
                        parm.targetId = groupsId;
                        parm.newsRelType = NewsRelType.futureCategory;

						String jsonParam2 = News_1_RetrieveHeadlineML_1.getCodeRequestJson2(maxCount, nRic);
                        JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl,
                                jsonParam2);
						processReutersNews(jsonNode2, parm);

                    });
			logger.info("handlerByFutrueTitle4Reuters end tickerChain : {}", nRic);
        } catch (Exception e) {
            logger.error("handlerByIndexTitle4Reuters index :{},error:{}", tickerChain, e);
        }

    }

    public void handlerByHomePage(String rtlang, String wlang) {
        String jsonParam2 = News_1_RetrieveHeadlineML_1.getHomePageTopicRequestJson(rtlang);
        JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam2);
		logger.info("handlerByHomePage start rtlang :{},wlang:{}", rtlang, wlang);
        ProcessNewsParm parm = new ProcessNewsParm();
        parm.targetId = 1;
        parm.exchangeId = 0;
        parm.regionId = 0;
        parm.newsRelType = NewsRelType.homePage;
		processReutersNews(jsonNode2, parm);

        String jsonParam3 = News_1_RetrieveHeadlineML_1.getHomePageReportRequestJson(rtlang);
        JsonNode jsonNode3 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam3);
		processReutersNews(jsonNode3, parm);
		logger.info("handlerByHomePage end rtlang :{},wlang:{}", rtlang, wlang);
    }

    public void handlerByForeignExchangeTitle4Reuters(Currency currency) {
        try {
            String code = Optional.ofNullable(currency).map(c -> c.getSymbol()).orElse(null);
            if (StringUtils.isBlank(code))
                return;
			// code = "CNY"
			String jsonParam2 = News_1_RetrieveHeadlineML_1.getCodeRequestJson2(maxCount, code + "/");
            JsonNode jsonNode2 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam2);
            ProcessNewsParm parm = new ProcessNewsParm();

            parm.targetId = currency.getId();
            parm.exchangeId = 0;
            parm.regionId = 0;
            parm.newsRelType = NewsRelType.currency;
			processReutersNews(jsonNode2, parm);

            /**
             * 中文新闻
             */
            String jsonParam3 = News_1_RetrieveHeadlineML_1.getCodeRequestJsonZH(maxCount, code + "/CN", false);
            JsonNode jsonNode3 = reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl, jsonParam3);
			processReutersNews(jsonNode3, parm);
			logger.info("handlerByForeignExchangeTitle4Reuters end currency : {}", currency.getAlias());
            // String jsonParam4 =
            // News_1_RetrieveHeadlineML_1.getCodeRequestJsonZH(maxCount, code +
            // "/", true);
            // JsonNode jsonNode4 =
            // reutersGuideUtil.getReutersData(News_1_RetrieveHeadlineML_1.restUrl,
            // jsonParam4);
            // processReutersNews(jsonNode4, parm, Constants.lang_zh_hant);

        } catch (Exception e) {
            logger.error("handlerByIndexTitle4Reuters index :{},error:{}", currency, e);
        }
    }

    private static class ProcessNewsParm {
        private Integer regionId;
        private Integer exchangeId;
        private Integer targetId;// tickeriD或者currencyId
        private NewsRelType newsRelType = NewsRelType.ticker;// 新闻关联种类,默认为ticker
    }

	private void processReutersNews(JsonNode jsonNode, ProcessNewsParm processNewsParm) {

		if (jsonNode == null || processNewsParm == null || processNewsParm.newsRelType == null
                || processNewsParm.targetId == null)
            return;
        JsonNode hls = jsonNode.at("/RetrieveHeadlineML_Response_1/HeadlineMLResponse/HEADLINEML/HL");
        if (hls == null || hls.isMissingNode())
            return;

        for (int i = 0; i < hls.size(); i++) {
			NewsInformation info = proudceNewsInfo(hls.get(i), processNewsParm.regionId);// produce(doc);
			// if (StringUtils.isBlank(info.getLanguage())) {
			// info.setLanguage(dfLang);
			// }
			if (Constants.lang_zh_hant.equals(info.getLanguage())) {// 繁体翻译简体
                NewsInformation info2 = new NewsInformation();
                BeanUtils.copyProperties(info, info2);
                if (info2.getTitle() != null)
                    info2.setTitle(ChineseHelper.convertToSimplifiedChinese(info2.getTitle()));
                if (info2.getSourceName() != null)
                    info2.setSourceName(ChineseHelper.convertToSimplifiedChinese(info2.getSourceName()));

                info2.setLanguage(Constants.lang_zh);
                info2.setUrl(info2.getUrl() + "_simplified");
                tikcerNewsProcessor.bindTargetNewsRecord(info2, processNewsParm.targetId, processNewsParm.newsRelType);
            }
            if (!StringUtils.startsWith(info.getLanguage(), "zh")) {
                info.setArticleClass(NewsInformation.ArticleClass.articleEn);
            }
            tikcerNewsProcessor.bindTargetNewsRecord(info, processNewsParm.targetId, processNewsParm.newsRelType);
        }

		// byTarget.setCollectNumber(hls.size());
		// stockConfigSupport.updateById(byTarget);
    }



	private NewsInformation proudceNewsInfo(JsonNode hl0, Integer regionId) {
        if (hl0 == null)
            return null;
        NewsInformation info = new NewsInformation();

        String urn = hl0.at("/ID").asText();
        String source = hl0.at("/PR").asText();
        String lang = hl0.at("/LN").asText();

        String title = hl0.at("/HT").asText();
        // System.out.println("********--->" + title);

        if (StringUtils.isNotBlank(title)) {
            title = StringUtils.removePattern(title, "(?i)<Origin [^<]{1,}>[^<]{1,}</Origin>");
            title = StringUtils.remove(title, "(with URL)");
            title = StringEscapeUtils.unescapeXml(title);// 转义特殊字符
            title = StringEscapeUtils.unescapeHtml4(title);// 转义特殊字符
        }
        info.setRegionId(regionId);
        try {

            // TimeZone.getTimeZone("GMT")

            Date newsTime = UtilDate.parse(hl0.at("/CT").asText(), Locale.US, TimeZone.getTimeZone("GMT"),
                    "yyyy-MM-dd'T'HH:mm:ssXXX");
            info.setNewsTime(newsTime);// // 2016-10-23T23:28:05+00:00
            // UtilDate.format(newsTime, Locale.US,
            // exchageTimezone.get(exchangeId),
            // "yyyy-MM-dd'T'HH:mm")
        } catch (Exception e) {
            logger.error("proudceNewsInfo StockReuteGuideHandler error:{}", e);
            // e.printStackTrace();
        }

        info.setUrl(StringUtils.stripToNull(urn));
        info.setTitle(StringUtils.stripToNull(title));

        if (Constants.lang_zh.equalsIgnoreCase(lang)) {
			// char[] a = title.toCharArray();
			// lang:
			// for (int i = 0; a != null && i < a.length; i++) {
			// if (ChineseHelper.isTraditionalChinese(a[i])) {
			lang = Constants.lang_zh_hant;
			// break lang;
			// }
			// }
        }

        info.setLanguage(StringUtils.lowerCase(lang));// zh/en
        info.setSourceName(StringUtils.stripToNull(source));
        info.setCollectSource(News.CollectSource.reutersGuide);
        return info;
    }

    /**
     * 仅支持单个story获取
     *
     * @param info0
     * @since 0.1.0
     */
    public void praseReutersNewsInfo(NewsInformation info0) {
        try {
            if (StringUtils.isBlank(info0.getUrl()))
                return;
            String nrid = StringUtils.substringBeforeLast(info0.getUrl(), "_simplified");

            String jsonParam2 = News_1_RetrieveStoryML_1.getUrnRequestJson(nrid);
            JsonNode jsonNode = reutersGuideUtil.getReutersData(News_1_RetrieveStoryML_1.restUrl, jsonParam2);
            boolean isCn = StringUtils.startsWith(info0.getLanguage(), "zh");
            Optional.ofNullable(jsonNode)//
                    .map(json0 -> json0.at("/RetrieveStoryML_Response_1/StoryMLResponse/STORYML/HL/0"))//
                    .ifPresent(json01 -> {
                        if (json01.isMissingNode())
                            return;

                        if (info0.getUrl().endsWith("_simplified"))
                            info0.setContent(ChineseHelper
                                    .convertToSimplifiedChinese(conversionContext(json01.at("/TE").asText(), isCn)));
                        else
                            info0.setContent(conversionContext(json01.at("/TE").asText(), isCn));
                        /**
                         * 后续填充其他字段，// 例如标签等
                         */
                    });

        } catch (Exception e) {
            logger.error(e);
        }

    }



	private @Autowired ReutersTopNewsListUitl reutersTopNewsListUitl;

	public void handlerTopNews2() {
		final Consumer<List<ReutersTopNewsListUitl.ReutersItem>> processList = items0 -> {
			if (items0 != null)
				items0.forEach(vv -> {

					NewsInformation info0 = new NewsInformation();

					info0.setRegionId(0);
					info0.setLanguage(Constants.lang_en);
					info0.setArticleClass(NewsInformation.ArticleClass.articleEn);
					info0.setCollectSource(News.CollectSource.reutersEn);
					info0.setTitle(vv.getTitle());
					info0.setUrl(vv.getLink());
					info0.setNewsTime(vv.getNewsTime());
					info0.setSourceName("reuters.com");
					info0.addMainPic(vv.getMainPic());
					info0.setMainProxy(true);
					info0.setSiteType(News.SiteType.PIRORIT);
					// info0.setForceReview(true);
					info0.setHomePage(vv.getHome());
					if (News.HomeType.frontPage.equals(info0.getHomePage())) {
						info0.getMainPic().setThumSize(450);
					}
					logger.info("handlerTopNews reuters.com....title:{},url:{}", info0.getTitle(), info0.getUrl());
					tikcerNewsProcessor.bindTargetNewsRecord(info0, 1, NewsRelType.homePage);
				});
		};
		List<ReutersTopNewsListUitl.ReutersItem> reutersItems = reutersTopNewsListUitl.parseNewsList();
		processList.accept(reutersItems);
		logger.info("handlerTopNews reuters.parseNewsList end!!!");

		List<ReutersTopNewsListUitl.ReutersItem> reutersItems2 = reutersTopNewsListUitl.parseBusinessList();
		processList.accept(reutersItems2);
		logger.info("handlerTopNews reuters.parseBusinessList end!!!");

		List<ReutersTopNewsListUitl.ReutersItem> reutersItems3 = reutersTopNewsListUitl.parseMarketsList();
		processList.accept(reutersItems3);
		logger.info("handlerTopNews reuters.parseMarketsList end!!!");
	}
    /**
     * 抓取topNews
     */
    public void handlerTopNews() {
        /**
         * 首页的专栏id
         */
        final String frontPages = "urn:newsml:reuters.com:20020923:SPDOC_119827232002";
        /**
         * 取有权限的cloumns
         */
        String jsonParam2 = GetTopNewsEntitledColumns_1.getRicRequestJson();
        JsonNode jsonNode2 = reutersGuideUtil.getReutersData(GetTopNewsEntitledColumns_1.restUrl, jsonParam2);
        final List<String> columns = new ArrayList<>();

        Optional.ofNullable(jsonNode2).map(v -> v.at("/GetTopNewsEntitledColumns_Response_1/Columns/Column"))
                .filter(v -> v != null && !v.isMissingNode()).ifPresent(v -> v.forEach(v0 -> {
            logger.info("handlerTopNews GetTopNewsEntitledColumns_Response_1 add name:{},columnsId:{}",
                    v0.at("/name").asText(), v0.at("/columnId").asText());
            Optional.ofNullable(StringUtils.stripToNull(v0.at("/columnId").asText()))
                    .ifPresent(v1 -> columns.add(v1));
        }));
        logger.info("handlerTopNews GetTopNewsEntitledColumns_Response_1 columns:{}", ArrayUtils.toString(columns));

        if (columns.isEmpty()) {
            logger.warn("handlerTopNews columns is empty!!!!");
            return;
        }

        /**
         * 解析新闻list
         */
        // reuter在item.size=1的情况下，使用map而不是array
        final Function<JsonNode, Iterable<JsonNode>> funcConverArrayNode = node -> {
            if (node == null || node.isMissingNode())
                return null;
            return node.isArray() ? node : Arrays.asList(node);
        };

        /**
         * 根据cloumns取信息
         */
        for (String columnId : columns) {
            String jsonParam3 = RetrieveTopNewsByColumnID_1.getRicRequestJson(columnId);
            JsonNode jsonNode3 = reutersGuideUtil.getReutersData(RetrieveTopNewsByColumnID_1.restUrl, jsonParam3);
            jsonNode3 = Optional.ofNullable(jsonNode3)
                    .map(v -> v
                            .at("/RetrieveTopNewsByColumnID_Response_1/Story/OtherContent/NewsML/NewsItem/NewsComponent/NewsComponent"))
                    .filter(v -> !v.isMissingNode()).orElse(null);

            int[] components = {0, 0};
            Optional.ofNullable(funcConverArrayNode.apply(jsonNode3)).ifPresent(v -> v.forEach(v0 -> {
                components[0]++;
                JsonNode newsItem = v0.at("/NewsItem/NewsComponent");
                if (newsItem == null || newsItem.isMissingNode())
                    return;
                NewsInformation info0 = new NewsInformation();
                info0.setSiteType(News.SiteType.NORMAL);
                info0.setHomePage(News.HomeType.top);
                info0.setRegionId(0);
                info0.setLanguage(Constants.lang_en);
                info0.setArticleClass(NewsInformation.ArticleClass.articleEn);
                info0.setCollectSource(News.CollectSource.reutersGuide);
				info0.setTitle(
						StringUtils.strip(//
								StringUtils.strip(newsItem.at("/NewsLines/HeadLine").asText(null), "*"),
								"-"));

                Optional.ofNullable(funcConverArrayNode.apply(newsItem.at("/NewsComponent/NewsComponent")))
                        .ifPresent(v2 -> v2.forEach(v3 -> {
                            String formalName = v3.at("/Role/@FormalName").asText("null");

                            if (StringUtils.containsIgnoreCase(formalName, "Text")) {
                                //  新闻内容 "@FormalName":"Main Text"
//                                String source = Optional
//                                        .ofNullable(v3.at("/NewsComponent/NewsLines/DateLine").asText(null))
//                                        .map(vn -> StringUtils.substringBetween(vn, "(", ")")).orElse("Reuters");
                                info0.setSourceName("Reuters");
                                // 新闻内容
                                Optional.ofNullable(funcConverArrayNode
                                        .apply(v3.at("/NewsComponent/ContentItem/Characteristics/Property")))
                                        .ifPresent(v4 -> genTopNewsUrlAndTime(v4, info0));
                                // 概要
                                info0.setSummary(StringUtils.substring(
                                        v3.at("/NewsComponent/ContentItem/DataContent/html/body/p").asText(), 0, 250));

                            } else if (StringUtils.containsIgnoreCase(formalName, "Related")) {
                                //关联新闻部分 "@FormalName":"Related Content"

                                Optional.ofNullable(funcConverArrayNode.apply(v3.at("/NewsComponent")))
                                        .ifPresent(v4 -> v4.forEach(v5 -> {
                                            NewsInformation.AssociatesNews associatesNews = info0.derivedAssociate();

                                            Optional.ofNullable(funcConverArrayNode.apply(v5.at("/NewsComponent/Characteristics/Property")))
													.ifPresent(v6 -> genTopNewsUrlAndTime(v6, associatesNews));
                                            //标题
                                            associatesNews.setTitle(v5.at("/NewsComponent/NewsLines/HeadLine").asText(null));
                                            // 概要
                                            associatesNews.setSummary(StringUtils.substring(
                                                    v5.at("/NewsComponent/ContentItem/DataContent/html/body/p").asText(), 0, 250));

                                            associatesNews.setSourceName("Reuters");
                                            if (NewsCcUtil.checkNewsInformation0(associatesNews)) {
                                                info0.addAssociatesNews(associatesNews);
                                            }

                                        }));

                            } else if (StringUtils.containsIgnoreCase(formalName, "Picture")) {
                                // 图片部分  "@FormalName":"Main Picture"
                                Optional.ofNullable(
                                        funcConverArrayNode.apply(v3.at("/NewsComponent/NewsComponent/ContentItem")))
                                        .ifPresent(v4 -> v4.forEach(v5 -> {
                                            String s0 = Optional.ofNullable(v5.at("/Characteristics/Property"))
                                                    .filter(v6 -> !v6.isMissingNode()).map(v7 -> {
                                                        for (int i = 0; i < v7.size(); i++) {
                                                            if ("PicType".equalsIgnoreCase(
                                                                    v7.get(i).at("/@FormalName").asText()))
                                                                return v7.get(i).at("/@Value").asText();
                                                        }
                                                        return null;
                                                    }).orElse(null);

                                            String s1_url = v5.at("/@Href").asText(null);
                                            if ("ViewImage".equalsIgnoreCase(s0)) {
                                                info0.getContextPic().setContextPic(s1_url);// 主图
                                                info0.getContextPic().setContextProxy(true);

                                                info0.addMainPic(s1_url);// 缩略图
                                                info0.setMainProxy(true);
                                                if (frontPages.equalsIgnoreCase(columnId)) {// 是front
                                                    // page
                                                    // 用大图模式
                                                    info0.getMainPic().setThumSize(450);
                                                    info0.setHomePage(News.HomeType.frontPage);
                                                }
                                            }
                                        }));
                            }

                        }));
                logger.info("handlerTopNews proc,info0:{},columnId:{}", info0.getUrl(), columnId);
                if (NewsCcUtil.checkNewsInformation0(info0)) {
                    if (info0.getMainPic().getMainPic().isEmpty()) {
                        logger.warn("handlerTopNews getMainPic null,info0:{},columnId:{}", info0.getUrl(), columnId);
                    }
                    components[1]++;
                    /*
                      英文标题的重复性判断
                     */
                    if (ChineseHelper.containsChinese(info0.getTitle()) || !repeatEnActuator.apply(info0.getTitle()))
                        tikcerNewsProcessor.bindTargetNewsRecord(info0, 1, NewsRelType.homePage);
					else
						logger.warn("repeatEnActuator true:{},{},{}", columnId, info0.getTitle(), info0.getUrl());
                }

            }));
            if (components[0] != components[1]) {
                logger.warn("handlerTopNews warn!!!!!!!,components is not equal");
            }

            logger.info("handlerTopNews columnId:{},components:{}", columnId, ArrayUtils.toString(components));

        }

    }

    private void genTopNewsUrlAndTime(Iterable<JsonNode> node, NewsInformation info) {
        node.forEach(v5 -> {
            String s0 = v5.at("/@FormalName").asText();
            String s1 = v5.at("/@Value").asText();
            if ("USN".equalsIgnoreCase(s0))
                info.setUrl(NewsCcUtil.TopNewsPrefix + s1);// "@Value":"nL8N1I75EP"
            else if ("ContentCreationDateAndTime".equalsIgnoreCase(s0))
                try {
                    info.setNewsTime(DateUtils.parseDate(s1, Locale.ENGLISH,
                            "EEE, dd MMM yyyy HH:mm:ss Z"));// "@Value":"Thu,
                    //  "@Value":"Tue, 13 Jun 2017 20:17:00 GMT"
                } catch (Exception e) {
                }
        });
    }

    private Function<String, Boolean> repeatEnActuator = new Function<String, Boolean>() {
        private List<String> titles = new LinkedList<>();

        @Override
        public Boolean apply(String title) {

            for (String title0 : titles) {
				if (SimilarDegreeByCos.getSimilarDegree(title, title0) > 0.9f) {
					return true;
				}
            }
            // 未重复
			if (titles.size() > 200)
                ((Queue) titles).poll();

            titles.add(title);
            return false;
        }
    };


    public void praseReutersTopNewsInfo(NewsInformation info0) {
        try {
            if (StringUtils.isBlank(info0.getUrl()))
                return;
            String storyId = StringUtils.substringAfter(info0.getUrl(), NewsCcUtil.TopNewsPrefix);

            String jsonParam = GetNewsStory_1.getRicRequestJson(storyId);
            JsonNode jsonNode = reutersGuideUtil.getReutersData(GetNewsStory_1.restUrl, jsonParam);

            Optional.ofNullable(jsonNode)//
                    .map(v -> v.at("/GetNewsStory_Response_1/Story/0"))//
                    .filter(v -> !v.isMissingNode()).ifPresent(v1 -> {
                // Optional.ofNullable(v1.at("/StoryProps/ItemMeta/Source").asText(null)).ifPresent(v2
                // -> info0.setSourceName("Reuters"));
                info0.setSourceName("Reuters");// 写死
                Optional.ofNullable(v1.at("/StoryProps/ContentMeta/Language/0"))
                        .map(v2 -> !v2.isMissingNode()
                                ? StringUtils.stripToNull(StringUtils.substringAfter(v2.asText(), "L:")) : null)
                        .ifPresent(v3 -> info0
                                .setLanguage(Constants.ReuterLangMaping.getOrDefault(v3, v3.toLowerCase())));

                boolean isCn = StringUtils.startsWithIgnoreCase(info0.getLanguage(), "zh");

                String value = Optional.ofNullable(v1.at("/StoryProps/ContentMeta/Slugline/Value").asText(null))
                        .map(v2 -> StringUtils.substringBetween(v2, "(", ")")).orElse("");

                if (StringUtils.containsAny(value, "REPEAT", "UPDATE"))//重复的新闻判定
                    info0.setContent(null);
                else
                    // 正文
                    Optional.ofNullable(v1.at("/HtmlContent").asText(null)).ifPresent(v2 -> {
                        info0.setContent(conversionTopContext(info0, v2, isCn));
                    });

            });

        } catch (Exception e) {
            logger.error(e);
        }

    }

	private static final String regex000 = "(?i)(<p>)?\\s*[（|\\(]Reporting\\s+by\\s+[^<]+?[\\)|）]\\s*(</p>)?\\s*$";
    private String conversionTopContext(NewsInformation info0, String context, boolean isCn) {
        if ("null".equalsIgnoreCase(context))
            return null;

        context = StringUtils.substringBetween(context, "</head><body>", "</body></html>");


        // 删除私有标签 <span id=\"x2\">nL8N1IN057</span>
        context = StringUtils.removePattern(context, "(?i)<span\\s+id=.{1,2}\\D+\\d+.{1,2}>.{0,4}n[0-9a-zA-Z]+</span>");

        // 删除结尾邮箱等联系方式,不能跨段落
        Matcher m0 = Pattern.compile("(?i)<p>\\s*[(]{2}.+?[)]{2}\\s*</p>\\s*$").matcher(context);
        if (m0.find() && StringUtils.countMatches(m0.group(), "<p>") == 1)
            context = StringUtils.substring(context, 0, m0.start());

        // 删除次结尾的 作者，报道,不能跨段落
        m0 = Pattern.compile("(?i)<p>\\s*[(]{1}.+?[)]{1}\\s*</p>\\s*$").matcher(context);
        if (m0.find() && StringUtils.countMatches(m0.group(), "<p>") == 1)
            context = StringUtils.substring(context, 0, m0.start());


        if (isCn) {
            // <p>****参考原文报导
            Matcher m = Pattern.compile("(?i)<p>\\*{4,}.{0,2}参考原文报导.+").matcher(context);
            if (m.find())
                context = StringUtils.substring(context, 0, m.start());
        } else {
            // 剔除开头的文字说明
            Matcher m = Pattern
                    .compile("(?i)<p>By\\s+[^<]+</p>\\s*<p><span>[^<]+\\d+[^<]+</span><span>\\s*-[^<]*</span>")
                    .matcher(context);
            if (m.find())
                context = StringUtils.substring(context, m.end());

            // 段落间隙处理
            context = StringUtils.replacePattern(context, "<br\\s*/>", "");
            context = StringUtils.replacePattern(context, "</p>\\s*<p>", "</p><div style='height: 17px;'></div><p>");
        }

        String[] imgTag = {""};
        // 主图
        if (StringUtils.isNotBlank(info0.getContextPic().getContextPic())) {
            Optional.ofNullable(httpClientUtil.getByteEntity(info0.getContextPic().getContextPic(),
                    info0.getContextPic().isContextProxy())).ifPresent(v -> {
                String contextPic = "2/" + DigestUtils.md5Hex(info0.getUrlMd5()).substring(8, 24) + ".jpg";
                try {
                    if (awsS3ClientUtil.uploadNewsPicture(contextPic, v)//
                            && awsUsS3ClientUtil.uploadNewsPicture(contextPic, v)) {
                        imgTag[0] = "<img src='../" + contextPic + "'   />";
                    }
                } catch (Exception e) {
                    logger.warn("conversionTopContext .upload contextPicutre", e);
                }

            });
        }
		if (isCn) {
			context = StringUtils.substringBeforeLast(context, "(完)");
		}
		context = StringUtils.removePattern(context, regex000);
        context = imgTag[0] + context;
        return StringUtils.stripToNull(context);
    }

    /**
     * @param args
     * @throws IOException
     */
    public static void main(String[] args) throws IOException {
		String context = "<div style='height: 17px;'></div><p>($1 = 3.6265 zlotys)</p><div style='height: 17px;'></div><p></p><div style='height: 17px;'></div>(Reporting by Agnieszka Barteczko; Editing by Sunil Nair）1";
		System.out.println(
				StringUtils.removePattern(context,
						"(?i)(<p>)?\\s*[（|\\(]Reporting\\s+by\\s+[^<]+?[\\)|）]\\s*(</p>)?\\s*$"));

		// String context2 = "<pre><p> June 21 (Reuters) - China Vanke Co Ltd <Origin
		// Href=\\\"QuoteRef\\\">000002.SZ</Origin> <Origin
		// Href=\\\"QuoteRef\\\">2202.HK</Origin> \\n * says Chairman Wang Shi no longer
		// nominated as director of \\nthe board \\n \\n \\n Source text in Chinese:
		// <Origin Href=\\\"Link\\\">http://bit2.ly/2sNYA1I</Origin> \\n Further company
		// coverage: <Origin Href=\\\"NewsSearch\\\">000002.SZ</Origin> <Origin
		// Href=\\\"NewsSearch\\\">2202.HK</Origin> \\n \\n (Reporting by Hong Kong
		// newsroom) \\n ((hongkong.newsroom@thomsonreuters.com; (8610) 6627-1277;
		// \\nReuters Messaging: min.zhang@thomsonreuters.com))</p></pre>";
		//
		// System.out.println(conversionContext(context2, false));
    }

    private static String conversionContext(String context, boolean isCN) {
        if ("null".equalsIgnoreCase(context))
            return null;

        // ****如欲參考原文報導，請點選 urn:newsml:reuters.com:*:nL3N1BS00J即可擷取瀏覽****
        /**
         * 不跨行删除
         */
        context = replaceContextAuthor(context);
        context = StringUtils.removePattern(context, "\\*{4,}[^\\n|\\r]{1,}\\*{4,}");

        context = StringUtils.removePattern(context, "(?i)<pre>|</pre>");
        context = replaceHref(context);
        context = StringUtils.removePattern(context, "\\({2}[^(]+@\\S+\\.+[^)]+\\){2}");
        context = replaceContextKeywords(context);
        /*
         * 替换明文的\n
		 */
        context = StringUtils.replace(context, "\\n", "\n");
        /*
         * 删除此处的换行
		 */
        context = StringUtils.replacePattern(context, "(?i)Further\\s+company\\s+coverage:\\s+",
                "Further company coverage:");

        context = replaceContentTitle(context);

        if (isCN) {
            context = StringUtils.replacePattern(context, "(\\n\\s*){2,}", "<br><div style='height:19px;'></div>");
            context = StringUtils.replacePattern(context, "\\n\\s{4,}", "<br><div style='height:19px;'></div>");
        } else {
            context = StringUtils.replacePattern(context, "(\\n\\s*){2,}", "<br><p style='height: 17px;'></p>");
            context = StringUtils.replacePattern(context, "\\n\\s{4,}", "<br><p style='height: 17px;'></p>");
        }

        context = StringUtils.replacePattern(context, "[_|-]{1,}\\s*\\n", "<br>");
        context = StringUtils.replacePattern(context, "\\n\\s*[_|-]{1,}", "<br>");
        context = StringUtils.replacePattern(context, "[>]{1,}\\s*\\n", "><br>");
        context = StringUtils.replacePattern(context, "\\n\\s*[<]{1,}", "<br><");
        context = StringUtils.replacePattern(context, "\\n", "");
        context = StringUtils.replacePattern(context, "\\n", "");
        context = StringUtils.replacePattern(context, "-{31,}", StringUtils.leftPad("", 30, "-"));
        context = StringUtils.replacePattern(context, "_{31,}", StringUtils.leftPad("", 30, "_"));

        /**
         * 新闻超链接去除
         */
        context = StringUtils.removePattern(context, "(?i)<origin href=\"StoryRef\">[^<]{1,}</origin>");
		context = StringUtils.removePattern(context, regex000);
        context = replaceBanks(context);

        // context = StringEscapeUtils.unescapeHtml4(context);// HTML转义
        context = StringUtils.replacePattern(context, "<br>\\s*<br>", "<br>");
        return StringUtils.stripToNull(context);
    }

    /**
     * 替换路透超链接
     *
     * @param context
     * @return
     * @since 0.1.0
     */
    private static String replaceHref(String context) {
        if (StringUtils.isBlank(context))
            return "";
        // <Origin
        // Href=\"Link\">https://cns.omxgroup.com/cds/DisclosureAttachmentServlet?messageAttachmentId=586160</Origin>
        Matcher m = Pattern.compile("(?i)<Origin[^<]{1,}Href[^<]{1,}>[^<]{1,}</Origin>").matcher(context);
        while (m.find()) {
            String origin = m.group();
            String href = StringUtils.substringBetween(origin, ">", "<");
            if (StringUtils.startsWithIgnoreCase(href, "http") || StringUtils.startsWithIgnoreCase(href, "www")) {
//            	http://bit.ly/2sNYA1I
				String target = StringUtils.containsIgnoreCase(href, "bit.") ? " target='_blank'" : "";
				String replace = "<a href='" + href + "'" + target + ">" + href + "</a>";
                context = StringUtils.replaceOnce(context, origin, replace);
            } else {
                context = StringUtils.replaceOnce(context, origin, "");
            }
        }
        return context;
    }

    /**
     * 去除多余空格
     *
     * @param context
     * @return
     * @since 0.1.0
     */
    private static String replaceBanks(String context) {
        if (StringUtils.isBlank(context))
            return null;
        // <Origin
        // Href=\"Link\">https://cns.omxgroup.com/cds/DisclosureAttachmentServlet?messageAttachmentId=586160</Origin>
        Matcher m = Pattern.compile("[^>\\s]{1}\\s{2,}").matcher(context);
        while (m.find()) {
            String origin = m.group();
            context = StringUtils.replaceOnce(context, origin, origin.substring(0, 1) + " ");
        }
        return context;
    }

    private static String replaceContentTitle(String context) {
        // Pattern pattern1 =
        // Pattern.compile("\\n\\s*[(^a-z)&\\S]{1,}\\n{1}\\s{1,}");
        Pattern pattern1 = Pattern.compile("\\n\\s{2,}[^a-z]{1,}\\S{1}\\s*\\n");
        Matcher matcher = pattern1.matcher(context);
        StringBuffer sb = new StringBuffer(context);
        List<Integer> tars = new ArrayList<>();

        while (matcher.find()) {
            tars.add(matcher.end());
            // System.out.println(matcher.group());
        }
        for (int i = tars.size() - 1; i >= 0; i--) {
            sb.replace(tars.get(i), tars.get(i), "<br>");
        }
        return sb.toString();
    }

    private static String replaceContextAuthor(String context) {
        String pattern = "\\(\\b(Reuters)\\b\\)\\s{1,}-\\s{1,}";
        Pattern pattern1 = Pattern.compile(pattern);
        Matcher matcher = pattern1.matcher(context);
        StringBuffer sb = new StringBuffer(context);
        List<Integer> tars = new ArrayList<>();

        while (matcher.find()) {
            tars.add(matcher.end());
        }
        if (!tars.isEmpty())
            sb.replace(0, tars.get(0), "");

        return sb.toString();
    }

    private static String replaceContextKeywords(String context) {
        String pattern = "\\s{1,}\\n\\b(Keywords)\\b";
        Pattern pattern1 = Pattern.compile(pattern);
        Matcher matcher = pattern1.matcher(context);
        StringBuffer sb = new StringBuffer(context);
        List<Integer> tars = new ArrayList<>();

        while (matcher.find()) {
            tars.add(matcher.start());
        }
        if (!tars.isEmpty())
            sb.replace(tars.get(tars.size() - 1), context.length(), "");

        return sb.toString();
    }
}

// "space":"preserve",
// "ID":"urn:newsml:onlinereport.com:20160919:nRTROPT20160919093147KCN11P0RR",
// "RE":1,
// "ST":"Usable",
// "CT":"2016-09-19T09:31:47-00:00",///// 09/06/2016 1:21 PM
// "RT":"2016-09-19T09:31:47-00:00",
// "LT":"2016-09-19T09:33:10-00:00",
// "PR":"onlinereport.com",
// "AT":"IDS",
// "UR":4,
// "LN":"en",
// "HT":"Thailand's Country Group gets $375 million financing through China Ping
// An Bank",
// "TY":"story",
// "PE":"2404 99999",
// "CO":"CGD.BK 000001.SZ BJCEG.UL",
// "TO":"REDV TH FINS 000001.SZ CMPNY BJCEG.UL OLASBUS_SPO ASIA CGD.BK BANK
// OLASBUS DBT LOA THA OLRTXT BACT TEXT CHN FIN BNK OLASBUS_US CN COEN",
// "NI":null,
// "TN":1,
// "PD":null
